# traffic flow summary by 5 minute
'''
target_cross(目标路口):
    需要提交预测的路口。（训练集和测试集中都没有任何交通流的信息）
relate_cross(相关路口):
    跟目标路口挨着或在同一条路上的，且在训练集和测试集中有交通流数据的路口(目标路口与相关路口的对应字典)。
pred_cross(预测路口):
    跟目标路口挨着或在同一条路上的，且在训练集和测试集中有交通流数据的路口(所有目标路口的相关路口的集合)。
similar_cross(相似路口):
    跟目标路口挨着或在同一条路上的路口。

'''

import numpy as np
import pandas as pd 
import os
import pickle as pkl


#1 import meta-data
pwd = os.getcwd()
train_path = pwd + "/data/trainCrossroadFlow/"
test_path = pwd + "/data/testCrossroadFlow/"
crossName = pd.read_csv(pwd+"/data/trainCrossroadFlow/crossroadName.csv",index_col=0)
roadNetList = pd.read_csv(pwd+"/data/trainCrossroadFlow/roadnet.csv",index_col=None)
submitList = pd.read_csv(pwd+"/data/testCrossroadFlow/submit_example.csv",index_col=None)


#3 construct road-net by adjacency list from roadnet.csv
roadNet = {}
for k in range(len(roadNetList)):
    if roadNetList.iloc[k]['uproadID']  not in roadNet:
        roadNet[roadNetList.iloc[k]['uproadID']] = []
    if roadNetList.iloc[k]['downroadID']  not in roadNet:
        roadNet[roadNetList.iloc[k]['downroadID']] = []
    if roadNetList.iloc[k]['downroadID']  not in roadNet[roadNetList.iloc[k]['uproadID']]:
        roadNet[roadNetList.iloc[k]['uproadID']].append(roadNetList.iloc[k]['downroadID'])
    if roadNetList.iloc[k]['uproadID']  not in roadNet[roadNetList.iloc[k]['downroadID']]:
        roadNet[roadNetList.iloc[k]['downroadID']].append(roadNetList.iloc[k]['uproadID'])
del(roadNetList)


#2 parse CrossName,
crossName=crossName[~crossName.index.duplicated(keep='first')] #del duplicated
crossName['road1'] = np.zeros([len(crossName),1])
crossName['road2'] = np.zeros([len(crossName),1])
crossName['crossroadID'] = crossName.index
crossName.index = range(len(crossName))
for k in range(len(crossName)):
    tmpStr = crossName.iloc[k]['crossroadName']
    tmpList = tmpStr[:-3].split('与')
    crossName.loc[k,'road1'] = tmpList[0]
    crossName.loc[k,'road2'] = tmpList[1]
# get similar cross
similar_cross = {}
for k in range(len(crossName)):
    if crossName.loc[k,'crossroadID'] not in similar_cross:
        similar_cross[crossName.loc[k,'crossroadID']] = []
    for x in range(k+1, len(crossName)):
        if crossName.loc[x,'crossroadID'] not in similar_cross:
            similar_cross[crossName.loc[x,'crossroadID']] = []
        if (crossName.loc[k,'road1'] in [crossName.loc[x,'road1'],crossName.loc[x,'road2']] \
        or crossName.loc[k,'road2'] in [crossName.loc[x,'road1'],crossName.loc[x,'road2']] ):
            if crossName.loc[x,'crossroadID'] not in similar_cross[crossName.loc[k,'crossroadID']]:
                similar_cross[crossName.loc[k,'crossroadID']].append(crossName.loc[x,'crossroadID'])
                similar_cross[crossName.loc[x,'crossroadID']].append(crossName.loc[k,'crossroadID'])
# add roadNet to similar_cross
for x in roadNet:
    if x not in similar_cross:
        similar_cross[x] = []
    for y in roadNet[x]:
        if y not in similar_cross[x]:
            print(x,y)
            similar_cross[x].append(y)



#4 import tain and test data, transform to 5-minute-flow
train_list = []
for fileName in os.listdir(train_path):
    if fileName[:5] == 'train':
        train_list.append(fileName)
train_list.sort(key=lambda x:int(x.split('_')[-1].split('.')[0]))

test_list = []
for fileName in os.listdir(test_path):
    if fileName[:5] == 'test_':
        test_list.append(fileName)
test_list.sort(key=lambda x:int(x.split('_')[-1].split('.')[0]))



#5 target Crossroad
target_cross = []
for k in range(len(submitList)):
    if submitList.iloc[k]['crossroadID'] not in target_cross:
        target_cross.append(submitList.iloc[k]['crossroadID'])


#6 get related-crossroad
relate_cross = {}
# discard for cross_road should be in 'trainCrossroadFlow' dataset
#for k in range(len(target_cross)):
#    relate_cross[target_cross[k]] = []
#    for x in roadNet[target_cross[k]]:
#        relate_cross[target_cross[k]].append(x)

# use pred_cross(86) estimate target_cross(35)
pred_cross = set()
for fileName in train_list:
    tmp = pd.read_csv(train_path+fileName,index_col=None)
    roadSet1 = set(tmp['crossroadID'])
    print(len(roadSet1))
    break
for fileName in test_list:
    testtmp = pd.read_csv(test_path+fileName,index_col=None)
    roadSet2 = set(testtmp['crossroadID'])
    print(len(roadSet2))
    break
for road in target_cross:
    relate_cross[road] = []
    is_in = 0
    for x in similar_cross[road]:
        if x in roadSet1 or x in roadSet2:
            pred_cross.add(x)
            relate_cross[road].append(x)
            is_in = is_in + 1
    print(road, is_in)
relate_cross[100041] = [100031]
pred_cross.add(100031)
print(len(pred_cross))


# ------------------------- scan all--slow--careful! ------------------------------

#7 summary of pred_cross (maybe random-walk later)
# generate time_series
#tr_rng = pd.date_range(start='2019-08-01 07:00:00',end='2019-08-19 18:55:00',freq = '5min')
#te_rng = pd.date_range(start='2019-08-20 07:00:00',end='2019-08-23 18:55:00',freq = '5min')
#tr_x = pd.DataFrame(np.zeros([len(tr_rng),1]),index=tr_rng)
#tr_y = pd.DataFrame(np.zeros([len(tr_rng),1]),index=tr_rng)
#te_x = pd.DataFrame(np.zeros([len(te_rng),1]),index=te_rng)
#tr_y = pd.DataFrame(np.zeros([len(te_rng),1]),index=te_rng)
rng = pd.date_range(start='2019-08-01 07:00:00',end='2019-08-23 18:55:00',freq = '5min')
flow = pd.DataFrame(np.zeros([len(rng),len(pred_cross)]),index=rng)
flow.columns = np.sort(list(pred_cross))

for fileName in train_list:
    print(fileName)
    tmp = pd.read_csv(train_path+fileName,index_col=None)
    for k in range(len(tmp)):
        crossroadID = tmp.loc[k,'crossroadID']
        if crossroadID in pred_cross:
            timestamp = tmp.loc[k,'timestamp']
            timestamp = timestamp[:-2] + '00'
            minute = int( np.floor(int(timestamp[-5:-3]) / 5) * 5 )
            timestamp = timestamp[:-5] + str(minute) + timestamp[-3:]
            flow.loc[timestamp, crossroadID] = flow.loc[timestamp, crossroadID] + 1
        
for fileName in test_list:
    print(fileName)
    tmp = pd.read_csv(test_path+fileName,index_col=None)
    for k in range(len(tmp)):
        crossroadID = tmp.loc[k,'crossroadID']
        if crossroadID in pred_cross:
            timestamp = tmp.loc[k,'timestamp']
            timestamp = timestamp[:-2] + '00'
            minute = int( np.floor(int(timestamp[-5:-3]) / 5) * 5 )
            timestamp = timestamp[:-5] + str(minute) + timestamp[-3:]
            flow.loc[timestamp, crossroadID] = flow.loc[timestamp, crossroadID] + 1
    

# submitList add timeStamp
from datetime import datetime
submitList['timeStamp'] = np.zeros([len(submitList),1])
for k in range(len(submitList)):
    timeStr = '2019-08-' + str(submitList.loc[k,'date'])+' '+str(submitList.loc[k,'timeBegin'])+':00'
    time_ = datetime.strptime(timeStr,'%Y-%m-%d %H:%M:%S')
    #timeStr = timeType.strftime('%Y-%m-%d %H:%M:%S')
    submitList.loc[k,'timeStamp'] = time_


# make data for model
#timeNoMap = {7:0,9:1,11:2,14:3,16:4,17:5} # No of time_range
#time_start = datetime.strptime('2019-08-01 07:55:04','%Y-%m-%d %H:%M:%S')


timeNo_ = pd.DataFrame(np.zeros([133*12,1])) # 0-22, onehot-23, np.zeros([288*19,1])
flow_day_ = pd.DataFrame(np.zeros([133*12,7]))
flow_minute_ = pd.DataFrame(np.zeros([133*12,6]))
flow_y_ = pd.DataFrame(np.zeros([133*12,6]))
import copy
data_tr = {}

count = 0
tmp_day = list(range(7))
tmp_minute = list(range(6))
tmp_y = list(range(6))
for cross in pred_cross:
    count = 0
    timeNo = copy.copy(timeNo_)
    flow_day = copy.copy(flow_day_)
    flow_minute = copy.copy(flow_minute_)
    flow_y = copy.copy(flow_y_)
    for k in range(len(flow)):
        time_ = flow.index[k]
        time_tmp = int( int(time_.hour)*100 + int(time_.minute) )
        if time_.day >= 8 and time_.day <= 19 and time_tmp >= 730 and time_tmp <= 1830:
            #print(time_)
            timeNo.loc[count] = time_.hour*2 + (1 if time_.minute>=30 else 0) - 15
            for ii in range(1,8):
                #print(k-288*ii)
                tmp_day[ii-1] = flow.iloc[k-288*ii][cross]
            flow_day.loc[count] = tmp_day
            for ii in range(1,7):
                tmp_minute[ii-1] = flow.iloc[k-ii][cross]
            flow_minute.loc[count] = tmp_minute
            for ii in range(0,6):
                tmp_y[ii] = flow.iloc[k+ii][cross]
            flow_y.loc[count] = tmp_y
            count = count + 1
    data_tr[cross] = [timeNo,flow_day,flow_minute,flow_y]


# save key-variables
pkl.dump([flow,relate_cross,crossName,target_cross,pred_cross,similar_cross,roadNet,submitList,data_tr],open(pwd+"/data.pkl",'wb'))




# ------------------------- TODO: reload flow and fullfill ZERO ------------------------------

[flow,relate_cross,crossName,target_cross,pred_cross,similar_cross,roadNet,submitList,data_tr]=pkl.load(open(pwd+"/data.pkl",'rb'))


timeNo_ = pd.DataFrame(np.zeros([133*12,1])) # 0-22, onehot-23, np.zeros([288*19,1])
flow_day_ = pd.DataFrame(np.zeros([133*12,7]))
flow_minute_ = pd.DataFrame(np.zeros([133*12,6]))
flow_y_ = pd.DataFrame(np.zeros([133*12,6]))
import copy
data_tr = {}
count = 0
tmp_day = list(range(7))
tmp_minute = list(range(6))
tmp_y = list(range(6))
for cross in pred_cross:
    count = 0
    timeNo = copy.copy(timeNo_)
    flow_day = copy.copy(flow_day_)
    flow_minute = copy.copy(flow_minute_)
    flow_y = copy.copy(flow_y_)
    for k in range(len(flow)):
        time_ = flow.index[k]
        time_tmp = int( int(time_.hour)*100 + int(time_.minute) )
        if time_.day >= 8 and time_.day <= 19 and time_tmp >= 730 and time_tmp <= 1830:
            #print(time_)
            timeNo.loc[count] = time_.hour*2 + (1 if time_.minute>=30 else 0) - 15


# save key-variables
pkl.dump([flow,relate_cross,crossName,target_cross,pred_cross,similar_cross,roadNet,submitList,data_tr],open(pwd+"/data.pkl",'wb'))









